# -*- coding: utf-8 -*-
import json

import scrapy
from waterwalk.items import *
from waterwalk.utils import *
import datetime
import urllib
import requests
from waterwalk.settings import USER_AGENTS,PROXY_LIST,CRAWL_START,CRAWL_END,CITYS
import re
import random
REG = {
    'number': r'(\d+)',
    'yuan': r'(\d+)元',
    'huxing': r'(\d)室(\d)?厅?(\d)?卫?',
    'floor': r'(\d+)\/(\d+)',
    'rent_type': r'(.租)',
    'payment': r'押(.)付(.)',
    'district_block': r'(.+?)\s+\-\s+(.+)',
    'coodinates': r'.*var\s+map\s?=\s?new\sBaiduDetailMap\((.*?)\,(.*?)\,.*',
    'city': r'.*var\s+city_name\s?=\s?\'(.*)\';',
    'price': r'(\d+).*?(\d+)',
    'norns': r'[^\r\n\s]+',
    'rm_url_params': r'(.*)\?.*',
    'bathroom': r'(独立?卫(生间)?)',
    'balcony': r'(独立?阳台)',
    'orientation': r'\s(.*)',
}

def get(url):
    return requests.get(url, headers= random.choice(USER_AGENTS)).json()

KEY = 'd306zoyjsyarp7ifhu67rjxn52tv0t20'
class AirbnbSpider(scrapy.Spider):
    name = 'airbnb'
    allowed_domains = ['airbnbchina.cn']
    custom_settings = {
        'DOWNLOAD_DELAY': 0.28,
    }
    list_url = "https://www.airbnbchina.cn/api/v2/explore_tabs?version=1.3.8&_format=for_explore_search_web&is_new_cards_experiment=true&luxury_pre_launch=false&query_understanding_enabled=false&show_groupings=true&supports_for_you_v3=true&metadata_only=false&is_standard_search=true&refinement_paths%5B%5D=%2Fhomes&selected_tab_id=home_tab&allow_override%5B%5D=&screen_size=medium&_intents=p1&currency=&locale=zh&"
    detail_url = "https://www.airbnbchina.cn/rooms/"

    def start_requests(self):
        for query in CITYS.keys():
            pages = list(range(CRAWL_START, CRAWL_END))
            random.shuffle(pages)
            for page in pages[:50]:
                params = {
                    "query": query,
                    "section_offset": page,
                    "key": KEY
                }
                url = self.list_url + urllib.parse.urlencode(params)
                request = scrapy.Request(url=url, callback=self.parse)
                request.meta['city'] = query
                yield request

    def parse(self, response):
        '''
        解析列表分页
        '''
        body_dict = json.loads(response.body.decode('utf-8'))
        list = []
        for l in body_dict['explore_tabs'][0]['sections']:
            if 'listings' in l:
                list.extend(l['listings'])
        for hs in list:
            detail = hs['listing']
            house_url = self.detail_url + str(detail['id'])
            # 日历
            cl_url = 'https://www.airbnbchina.cn/api/v2/calendar_months?_format=with_conditions&count=4&currency=CNY&locale=zh&'
            params = {
                "listing_id": detail['id'],
                "key": KEY
            }
            url = cl_url + urllib.parse.urlencode(params)
            request = scrapy.Request(url, self.calendar)

            house = HouseItem()
            house['collection'] = get_collection_name(response.meta['city'])
            house['title'] = detail['name']
            house['room_num'] = detail['bedrooms']
            house['bed_num'] = detail['beds']
            house['bathroom_num'] = detail['bathrooms']
            house['max_live_people'] = detail['person_capacity']
            room_type = detail['room_type']
            # 独立房间/整套
            if room_type.find('独立') != -1:
                house['room_type'] = 1
            else:
                house['room_type'] = 2

            house['review_num'] = detail['reviews_count']
            try:
                house['rating'] = detail['star_rating']
            except Exception:
                house['rating'] = 0
            house['pictures'] = detail['picture_urls']
            house['city'] = detail['city']
            # longti,lati
            house['longi'] = detail['lng']
            house['lati'] = detail['lat']
            house['position'] = {
                'type': 'Point',
                'coordinates': [float(house['longi']), float(house['lati'])]
            }

            house['uniqe_key'] = detail['id']
            # date and unique_key
            house['crawl_date'] = datetime.datetime.now()
            # collection_name

            house['source_url'] = house_url
            # 房东
            house['owner_id'] = detail['user']['id']
            house['price'] = hs['pricing_quote']['rate_with_service_fee']['amount']
            request.meta['house'] = house
            yield request

    # def parse_page(self, response):
    #     """
    #     普通页面规则
    #     :param response:
    #     :param house:
    #     :return:
    #     """
    #     house = HouseItem()
    #     house['collection'] = get_collection_name(response.meta['city'])
    #     house['title'] = response.css('h1._tpbrp::text').extract_first()
    #     house['content'] = trim(''.join(response.css('p._6z3til span span::text').extract()))
    #     huxing = response.css('div._36rlri span._eamm1ge::text').re(r'(\d)')
    #     # 卧室、床、卫生间、最大入住人数
    #     if len(huxing) == 3:
    #         huxing.insert(0,1)
    #     house['room_num'] = int(huxing[0])
    #     house['bed_num'] = int(huxing[1])
    #     house['bathroom_num'] = int(huxing[2])
    #     house['max_live_people'] = int(huxing[3])
    #
    #     exclusive_desc = response.css('div._79dbpfm div._u6igkd7::text').extract_first()
    #     # 独立房间/整套
    #     if exclusive_desc.find('独立') != -1:
    #         house['room_type'] = 1
    #     else:
    #         house['room_type'] = 2
    #
    #     #评价
    #     house['review_num'] = int(response.css('span._1cy09umr span::text').re_first(r'(\d+)'))
    #     house['rating'] = int(response.css('div._16tcko6 span::attr(aria-label)').re_first(r'(\d)'))
    #     house['city'] = response.css('div._190019zr::text').extract_first().split(r',')[0]
    #     house['pictures'] = response.css('div._3nukz4e img::attr(src)').extract()
    #
    #     #价格
    #
    #     # longti,lati
    #     house['longi'] = response.css('body').re_first(r'lng":(\d+\.\d+)')
    #     house['lati'] = response.css('body').re_first(r'lat":(\d+\.\d+)')
    #     house['position'] = {
    #         'type': 'Point',
    #         'coordinates': [float(house['longi']), float(house['lati'])]
    #     }
    #
    #     house['uniqe_key'] = int(response.url.split(r'/')[-1])
    #     # date and unique_key
    #     house['crawl_date'] = datetime.datetime.now()
    #     # collection_name
    #
    #     house['source_url'] = response.url
    #
    #     # 房东
    #     house['owner_id'] = int(response.css('div._11lszbw a._110nrr2::attr(href)').extract_first().split(r'/')[-1])
    #
    #     # 价格
    #     price_url = 'https://www.airbnbchina.cn/api/v2/pdp_listing_booking_details?_format=for_web_dateless&currency=CNY&locale=zh&'
    #     params = {
    #         "listing_id": house['uniqe_key'],
    #         "key": KEY
    #     }
    #     url = price_url + urllib.parse.urlencode(params)
    #     request = scrapy.Request(url,self.price)
    #     request.meta['house'] = house
    #     yield request
    #
    # def price(self,response):
    #     house = response.meta['house']
    #     body_dict = json.loads(response.body.decode('utf-8'))
    #     house['price'] = body_dict['pdp_listing_booking_details'][0]['p3_display_rate']['amount']
    #     # 日历
    #     cl_url = 'https://www.airbnbchina.cn/api/v2/calendar_months?_format=with_conditions&count=4&currency=CNY&locale=zh&'
    #     params = {
    #         "listing_id": house['uniqe_key'],
    #         "key": KEY
    #     }
    #     url = cl_url + urllib.parse.urlencode(params)
    #     request = scrapy.Request(url,self.calendar)
    #     request.meta['house'] = house
    #     yield request

    def calendar(self, response):
        house = response.meta['house']
        body_dict = json.loads(response.body.decode('utf-8'))
        house['calendar'] = [{'month': m['month'], 'days': m['days']} for m in body_dict['calendar_months']]
        # 评价
        rv_url = 'https://www.airbnbchina.cn/api/v2/reviews?currency=CNY&locale=zh&role=guest&_format=for_p3&_limit=100&_offset=0&_order=language_country&'
        params = {
            "listing_id": house['uniqe_key'],
            "key": KEY
        }
        url = rv_url + urllib.parse.urlencode(params)
        request = scrapy.Request(url,self.review)
        request.meta['house'] = house
        yield request

    def review(self, response):
        house = response.meta['house']
        body_dict = json.loads(response.body.decode('utf-8'))
        house['reviews'] = [{'comments': r['comments'], 'date': r['created_at'], 'rating': r['rating']} for r in body_dict['reviews']]
        # 房东
        owner_url = 'https://www.airbnbchina.cn/users/show/' + str(house['owner_id'])
        url = owner_url
        request = scrapy.Request(url, self.owner)
        request.meta['house'] = house
        yield request

    def owner(self, response):
        house = response.meta['house']
        try:
            house['owner_review_num'] = int(response.css('#reviews h2 small::text').re_first(r'(\d+)'))
            house['owner_house_num'] = int(response.css('h2.space-3 small::text').re_first(r'(\d+)'))
            house['owner_des'] = response.css('div.space-top-2 p::text').extract_first()
            house['owner_reg_date'] = trim(response.css('div.space-top-2 span.text-normal::text').extract_first())
        except Exception:
            pass
        return house